from __future__ import print_function

import math

from IPython import display
from matplotlib import cm
from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from sklearn import metrics
import tensorflow as tf
from tensorflow.python.data import Dataset
from tensorflow.python.framework import dtypes
import seaborn as sns
import os
import glob
from my.common.imgFun import *

# 从数据集中 筛选(加工)特征
def preprocess_features(california_housing_dataframe):
    """从数据集中 筛选(加工)特征
    Prepares input features from California housing data set.


    Args:
      california_housing_dataframe: A Pandas DataFrame expected to contain data
        from the California housing data set.
    Returns:
      A DataFrame that contains the features to be used for the model, including
      synthetic features.
    """
    selected_features = california_housing_dataframe[
        ["latitude",
         "longitude",
         "housing_median_age",
         "total_rooms",
         "total_bedrooms",
         "population",
         "households",
         "median_income"]]
    processed_features = selected_features.copy()
    # Create a synthetic feature. 创建一个组合特征(人均房屋数)
    processed_features["rooms_per_person"] = (
            california_housing_dataframe["total_rooms"] /
            california_housing_dataframe["population"])
    return processed_features


# 从数据集中 筛选(加工)结果(目标)
def preprocess_targets(california_housing_dataframe):
    """从数据集中 筛选(加工)结果(目标)
    Prepares target features (i.e., labels) from California housing data set.
    Args:
      california_housing_dataframe: A Pandas DataFrame expected to contain data
        from the California housing data set.
    Returns:
      A DataFrame that contains the target feature.
    """
    output_targets = pd.DataFrame()
    # Scale the target to be in units of thousands of dollars. 结果集单位压缩到千美元
    output_targets["median_house_value"] = (
            california_housing_dataframe["median_house_value"] / 1000.0)
    return output_targets


def construct_feature_columns(input_features):
    """ 把特征数据集转换为tf的特征类型集合
    Construct the TensorFlow Feature Columns.

    Args:
      input_features: The names of the numerical input features to use.
    Returns:
      A set of feature columns
    """
    # 都是数字列
    return set([tf.feature_column.numeric_column(my_feature)
                for my_feature in input_features])


def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):
    """
    生成训练用的输入函数
    Trains a neural net regression model.
    小批量随机梯度下降法（小批量 SGD）是介于全批量迭代与 SGD 之间的折衷方案。小批量通常包含 10-1000 个随机选择的样本。小批量 SGD 可以减少 SGD 中的杂乱样本数量，但仍然比全批量更高效。

    Args:
      features: pandas DataFrame of features
      targets: pandas DataFrame of targets
      batch_size: Size of batches to be passed to the model 批次大小(小批量训练)
      shuffle: True or False. Whether to shuffle the data. 打乱数据,如果指定，重新洗牌数据。
      num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely 重复次数
    Returns:
      Tuple of (features, labels) for next data batch
    """

    # Convert pandas data into a dict of np arrays. 转换特征为np矩阵
    features = {key: np.array(value) for key, value in dict(features).items()}

    # Construct a dataset, and configure batching/repeating. 构造数据集，并配置批处理/重复。
    ds = Dataset.from_tensor_slices((features, targets))  # warning: 2GB limit
    ds = ds.batch(batch_size).repeat(num_epochs)

    # Shuffle the data, if specified. 如果指定，重新洗牌数据。
    if shuffle:
        ds = ds.shuffle(10000)

    # Return the next batch of data. 返回下一批数据。
    features, labels = ds.make_one_shot_iterator().get_next()
    return features, labels


def train_nn_regression_model(
        learning_rate,
        steps,
        batch_size,
        hidden_units,
        training_examples,
        training_targets,
        validation_examples,
        validation_targets):
    """训练一个神经网络
    Trains a neural network regression model.

    In addition to training, this function also prints training progress information,
    as well as a plot of the training and validation loss over time.

    Args:
      learning_rate: A `float`, the learning rate.
      steps: A non-zero `int`, the total number of training steps. A training step
        consists of a forward and backward pass using a single batch.
      batch_size: A non-zero `int`, the batch size.
      hidden_units: A `list` of int values, specifying the number of neurons in each layer.
            使用 hidden_units 定义神经网络的结构。hidden_units 参数会创建一个整数列表，其中每个整数对应一个隐藏层，表示其中的节点数。以下面的赋值为例：
            hidden_units=[3,10]
            上述赋值为神经网络指定了两个隐藏层：
            第一个隐藏层包含 3 个节点。
            第二个隐藏层包含 10 个节点。
            如果我们想要添加更多层，可以向该列表添加更多整数。例如，hidden_units=[10,20,30,40] 会创建 4 个分别包含 10、20、30 和 40 个单元的隐藏层。
            默认情况下，所有隐藏层都会使用 ReLu 激活函数，且是全连接层。

      training_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for training.
      training_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for training.
      validation_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for validation.
      validation_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for validation.

    Returns:
      A `DNNRegressor` object trained on the training data.
    """

    periods = 10
    steps_per_period = steps / periods

    # Create a DNNRegressor object.
    # 创建梯度下降优化器
    my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
    # 这里的clip_by_norm是指对梯度进行裁剪，通过控制梯度的最大范式，防止梯度爆炸的问题，是一种
    # 比较常用的梯度规约的方式，为了安全起见，我们还会通过 clip_gradients_by_norm 将梯度裁剪应用到我们的优化器。梯度裁剪可确保梯度大小在训练期间不会变得过大，梯度过大会导致梯度下降法失败。
    my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
    # 构造神经网络实例
    dnn_regressor = tf.estimator.DNNRegressor(
        feature_columns=construct_feature_columns(training_examples),
        hidden_units=hidden_units,
        optimizer=my_optimizer
    )

    # Create input functions. 创建输入函数(训练用, 小批量随机梯度下降法（小批量 SGD）)
    training_input_fn = lambda: my_input_fn(training_examples,
                                            training_targets["median_house_value"],
                                            batch_size=batch_size)
    # 创建输入函数(预测用 重复次数1不重复)(预测训练集)
    predict_training_input_fn = lambda: my_input_fn(training_examples,
                                                    training_targets["median_house_value"],
                                                    num_epochs=1,
                                                    shuffle=False)
    # 创建输入函数(预测用 重复次数1不重复)(预测测试集)
    predict_validation_input_fn = lambda: my_input_fn(validation_examples,
                                                      validation_targets["median_house_value"],
                                                      num_epochs=1,
                                                      shuffle=False)

    # Train the model, but do so inside a loop so that we can periodically assess 训练模型，但是要在一个循环中进行，这样我们就可以周期性地进行评估
    # loss metrics. 损失指标。
    print("Training model...")
    print("RMSE (on training data):")
    training_rmse = []
    validation_rmse = []
    for period in range(0, periods):
        # Train the model, starting from the prior state. 训练模型，从之前的状态开始。
        dnn_regressor.train(
            input_fn=training_input_fn,
            steps=steps_per_period
        )
        # Take a break and compute predictions. 预测训练集
        training_predictions = dnn_regressor.predict(input_fn=predict_training_input_fn)
        training_predictions = np.array([item['predictions'][0] for item in training_predictions])
        # 预测测试集
        validation_predictions = dnn_regressor.predict(input_fn=predict_validation_input_fn)
        validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])

        # Compute training and validation loss. 计算平方损失
        training_root_mean_squared_error = math.sqrt(
            metrics.mean_squared_error(training_predictions, training_targets))
        validation_root_mean_squared_error = math.sqrt(
            metrics.mean_squared_error(validation_predictions, validation_targets))
        # Occasionally print the current loss.
        print("  period %02d : %0.2f" % (period, training_root_mean_squared_error))
        # Add the loss metrics from this period to our list. 将此期间的损失指标添加到我们的列表中。
        training_rmse.append(training_root_mean_squared_error)
        validation_rmse.append(validation_root_mean_squared_error)
    print("Model training finished.")

    # Output a graph of loss metrics over periods. 输出各时期的损失率图表。
    plt.ylabel("RMSE")
    plt.xlabel("Periods")
    plt.title("Root Mean Squared Error vs. Periods")
    plt.tight_layout()
    plt.plot(training_rmse, label="training")
    plt.plot(validation_rmse, label="validation")
    plt.legend()

    print("Final RMSE (on training data):   %0.2f" % training_root_mean_squared_error)
    print("Final RMSE (on validation data): %0.2f" % validation_root_mean_squared_error)

    return dnn_regressor


# ------------------------------------------- dnn, 神经网络相关

def parse_labels_and_features(dataset, featuresColumnLength=30000, scaling=1):
    """
    提取标签和特征
    Extracts labels and features.

    This is a good place to scale or transform the features if needed.

    Args:
      dataset: A Pandas `Dataframe`, containing the label on the first column and
        monochrome pixel values on the remaining columns, in row major order.
    Returns:
      A `tuple` `(labels, features)`:
        labels: A Pandas `Series`.
        features: A Pandas `DataFrame`.
    """
    # 提取标签
    labels = dataset[0]

    # 提取特征 DataFrame.loc index ranges are inclusive at both ends.
    features = dataset.loc[:, 1:featuresColumnLength]
    # Scale the data to [0, 1] by dividing out the max value, 255.
    # 通过除以最大值255 使得数据缩放到[0,1]之间
    if scaling != 1:
        features = features / scaling

    return labels, features


def create_training_input_fn(features, labels, batch_size, sumRows=743, num_epochs=None, shuffle=True):
    """ 创建训练的函数,调用输出一批次的 (特征, 标签)
    A custom input_fn for sending MNIST data to the estimator for training.

    Args:
      features: The training features.
      labels: The training labels.
      batch_size: Batch size to use during training.

    Returns:
      A function that returns batches of training features and labels during
      training.
      :param sumRows: 数据总行数
    """

    def _input_fn(num_epochs=None, shuffle=True):
        # Input pipelines are reset with each call to .train(). To ensure model
        # gets a good sampling of data, even when number of steps is small, we
        # shuffle all the data before creating the Dataset object
        idx = np.random.permutation(features.index)
        raw_features = {"pixels": features.reindex(idx)}
        raw_targets = np.array(labels[idx])

        ds = Dataset.from_tensor_slices((raw_features, raw_targets))  # warning: 2GB limit
        ds = ds.batch(batch_size).repeat(num_epochs)

        if shuffle:
            ds = ds.shuffle(sumRows)

        # Return the next batch of data.
        feature_batch, label_batch = ds.make_one_shot_iterator().get_next()
        return feature_batch, label_batch

    return _input_fn


def create_predict_input_fn(features, batch_size, labels=None):
    """ 创建预测用的函数,调用输出一批次的 (特征, 标签)
    A custom input_fn for sending mnist data to the estimator for predictions.

    Args:
      features: The features to base predictions on.
      labels: The labels of the prediction examples.可选字段

    Returns:
      A function that returns features and labels for predictions.
    """

    def _input_fn():
        raw_features = {"pixels": features.values}
        raw_targets = np.array(labels)

        ds = Dataset.from_tensor_slices((raw_features, raw_targets))  # warning: 2GB limit
        ds = ds.batch(batch_size)

        # Return the next batch of data.
        feature_batch, label_batch = ds.make_one_shot_iterator().get_next()
        return feature_batch, label_batch

    return _input_fn

# 训练函数
def train_DNN_Classifier_model(
        learning_rate,
        steps,
        batch_size,
        hidden_units,
        training_examples,
        training_targets,
        validation_examples,
        validation_targets,
        n_classes=2,
        sumRows=743,
        featuresColumnLength=30000,
        periods = 10,
        dropout=0.05,
        model_dir=None
):
    """Trains a linear classification model for the MNIST digits dataset.

    In addition to training, this function also prints training progress information,
    a plot of the training and validation loss over time, and a confusion
    matrix.

    Args:
      learning_rate: An `int`, the learning rate to use.
      steps: A non-zero `int`, the total number of training steps. A training step
        consists of a forward and backward pass using a single batch.
      batch_size: A non-zero `int`, the batch size.
      training_examples: A `DataFrame` containing the training features.
      training_targets: A `DataFrame` containing the training labels.
      validation_examples: A `DataFrame` containing the validation features.
      validation_targets: A `DataFrame` containing the validation labels.

    Returns:
      The trained `LinearClassifier` object.
      :param periods: periods=0时,直接返回classifier
      :param model_dir: 模型地址
      :param dropout: 早停法中的配置,降低拟合度
      :param featuresColumnLength: csv 中特征占的列数
      :param sumRows:  csv 总共的数据量
      :param n_classes: 分类个数
    """
    periods = periods

    if periods != 0:
        steps_per_period = steps / periods
        # Create the input functions.
        predict_training_input_fn = create_predict_input_fn(
            training_examples, batch_size, training_targets)
        predict_validation_input_fn = create_predict_input_fn(
            validation_examples, batch_size, validation_targets)
        training_input_fn = create_training_input_fn(
            training_examples, training_targets, batch_size, sumRows=sumRows)

    # Create a LinearClassifier object.
    my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)
    my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)

    classifier = tf.estimator.DNNClassifier(
        hidden_units=hidden_units,
        feature_columns=[
            tf.feature_column.numeric_column('pixels', shape=featuresColumnLength, dtype=dtypes.float32)],
        # feature_columns=construct_feature_columns(), 这里用set是错误的
        # 输出
        n_classes=n_classes,
        optimizer=my_optimizer,
        dropout=dropout,
        config=tf.estimator.RunConfig(keep_checkpoint_max=1),
        model_dir=model_dir
    )
    if periods == 0:
        return classifier

    # Train the model, but do so inside a loop so that we can periodically assess
    # loss metrics.
    print("Training model...")
    print("LogLoss error (on validation data):")
    training_errors = []
    validation_errors = []
    for period in range(0, periods):
        # Train the model, starting from the prior state.
        classifier.train(
            input_fn=training_input_fn,
            steps=steps_per_period
        )

        # Take a break and compute probabilities.
        training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))
        training_probabilities = np.array([item['probabilities'] for item in training_predictions])
        training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])
        training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id, n_classes, dtype="int64")
        # 计算损失时targets也的to_categorical,矩阵化
        training_targets_one_hot = tf.keras.utils.to_categorical(training_targets, n_classes, dtype="int64")
        validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))
        validation_probabilities = np.array([item['probabilities'] for item in validation_predictions])
        validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])
        validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id, n_classes, dtype="int64")
        # 计算损失时targets也的to_categorical,矩阵化
        validation_targets_one_hot = tf.keras.utils.to_categorical(validation_targets, n_classes, dtype="int64")
        # Compute training and validation errors.
        # training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)
        training_log_loss = metrics.log_loss(training_targets_one_hot, training_pred_one_hot)
        # validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)
        validation_log_loss = metrics.log_loss(validation_targets_one_hot, validation_pred_one_hot)
        # Occasionally print the current loss.
        print("  period %02d : %0.2f" % (period, validation_log_loss))
        # Add the loss metrics from this period to our list.
        training_errors.append(training_log_loss)
        validation_errors.append(validation_log_loss)
    print("Model training finished.")
    # Remove event files to save disk space.
    _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))

    # Calculate final predictions (not probabilities, as above).
    final_predictions = classifier.predict(input_fn=predict_validation_input_fn)
    final_predictions = np.array([item['class_ids'][0] for item in final_predictions])

    accuracy = metrics.accuracy_score(validation_targets, final_predictions)
    print("Final accuracy (on validation data): %0.2f" % accuracy)

    # Output a graph of loss metrics over periods.
    plt.ylabel("LogLoss")
    plt.xlabel("Periods")
    plt.title("LogLoss vs. Periods")
    plt.plot(training_errors, label="training")
    plt.plot(validation_errors, label="validation")
    plt.legend()
    plt.show()

    # Output a plot of the confusion matrix.
    cm = metrics.confusion_matrix(validation_targets, final_predictions)
    # Normalize the confusion matrix by row (i.e by the number of samples
    # in each class).
    cm_normalized = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
    ax = sns.heatmap(cm_normalized, cmap="bone_r")
    ax.set_aspect(1)
    plt.title("Confusion matrix")
    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.show()

    return classifier

# 开始预测

def predictImgUrl(imgUrl, classifier):
    ''' 预测单个图片的函数

    :param imgUrl:
    :return:
    '''
    dfData = pd.DataFrame([imgToNpArray(imgUrl)])
    dfPredict = create_predict_input_fn(dfData, 1, [3])
    dfPreResult = list(classifier.predict(input_fn=dfPredict))
    printMy(dfPreResult)
    dfPreProbabilities = np.array([item['probabilities'] for item in dfPreResult])
    dfPreClassId = np.array([item['class_ids'][0] for item in dfPreResult])
    printMy("dfPreProbabilities:", dfPreProbabilities, "dfPreClassId", dfPreClassId)
    return dfPreResult